{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ai/tool/bin/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ai/tool/bin/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-947765a5574c>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/ai/tool/bin/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/ai/tool/bin/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/ai/tool/bin/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/ai/tool/bin/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ./data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/ai/tool/bin/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = './data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "添加一个500个节点的隐层,使用sigmoid激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "#隐层的weight不能init为tf.zeros，否则算出的结果很差\n",
    "w1 = tf.Variable(tf.truncated_normal([784,500],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([500,10],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer1,w2)+b2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-bf86c3447efc>:11: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1475\n",
      "0.8935\n",
      "0.9158\n",
      "0.9252\n",
      "0.9339\n",
      "0.9378\n",
      "0.9426\n",
      "0.9464\n",
      "0.9482\n",
      "0.9562\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔300个循环打印一次结果\n",
    "    if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9571\n"
     ]
    }
   ],
   "source": [
    "  # Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加一个隐层后结果大副提升，再加一层试试\n",
    "添加两个隐层，第一层500个节点，第二层300个节点,都使用sigmoid激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "w1 = tf.Variable(tf.truncated_normal([784,500],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([500,300],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1,w2)+b2)\n",
    "w3 = tf.Variable(tf.truncated_normal([300,10 ],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer2, w3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0854\n",
      "0.8796\n",
      "0.9099\n",
      "0.9154\n",
      "0.9292\n",
      "0.9359\n",
      "0.9419\n",
      "0.9403\n",
      "0.9465\n",
      "0.9444\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔300个循环打印一次结果\n",
    "    if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9493\n"
     ]
    }
   ],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加两个隐层，经过多次计算，发现结果相差无几，所以决定将隐层数设为三层，同时调整神经元数量看看差别如何"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "enum1 = 300\n",
    "enum2 = 150\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "w1 = tf.Variable(tf.truncated_normal([784,enum1],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([enum1]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([enum1,enum2],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([enum2]) + 0.1)\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1,w2)+b2)\n",
    "w3 = tf.Variable(tf.truncated_normal([enum2,10 ],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer2, w3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1011\n",
      "0.868\n",
      "0.9129\n",
      "0.9183\n",
      "0.9273\n",
      "0.9327\n",
      "0.9383\n",
      "0.9441\n",
      "0.9475\n",
      "0.9513\n",
      "0.955\n"
     ]
    }
   ],
   "source": [
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1028\n",
      "0.8882\n",
      "0.9074\n",
      "0.9212\n",
      "0.9296\n",
      "0.9357\n",
      "0.9325\n",
      "0.9395\n",
      "0.9492\n",
      "0.9494\n",
      "0.9508\n"
     ]
    }
   ],
   "source": [
    "enum1 = 400\n",
    "enum2 = 200\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "w1 = tf.Variable(tf.truncated_normal([784,enum1],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([enum1]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([enum1,enum2],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([enum2]) + 0.1)\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1,w2)+b2)\n",
    "w3 = tf.Variable(tf.truncated_normal([enum2,10 ],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer2, w3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0974\n",
      "0.8904\n",
      "0.9124\n",
      "0.9134\n",
      "0.9223\n",
      "0.9213\n",
      "0.9412\n",
      "0.9439\n",
      "0.9449\n",
      "0.9517\n",
      "0.9515\n"
     ]
    }
   ],
   "source": [
    "enum1 = 600\n",
    "enum2 = 300\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "w1 = tf.Variable(tf.truncated_normal([784,enum1],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([enum1]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([enum1,enum2],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([enum2]) + 0.1)\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1,w2)+b2)\n",
    "w3 = tf.Variable(tf.truncated_normal([enum2,10 ],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer2, w3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1032\n",
      "0.8635\n",
      "0.9057\n",
      "0.9257\n",
      "0.9304\n",
      "0.9322\n",
      "0.937\n",
      "0.94\n",
      "0.9493\n",
      "0.9505\n",
      "0.9544\n"
     ]
    }
   ],
   "source": [
    "enum1 = 700\n",
    "enum2 = 300\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "w1 = tf.Variable(tf.truncated_normal([784,enum1],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([enum1]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([enum1,enum2],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([enum2]) + 0.1)\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1,w2)+b2)\n",
    "w3 = tf.Variable(tf.truncated_normal([enum2,10 ],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer2, w3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1028\n",
      "0.8969\n",
      "0.9115\n",
      "0.9211\n",
      "0.9317\n",
      "0.9375\n",
      "0.9406\n",
      "0.9393\n",
      "0.9501\n",
      "0.9526\n",
      "0.9518\n"
     ]
    }
   ],
   "source": [
    "enum1 = 800\n",
    "enum2 = 400\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "w1 = tf.Variable(tf.truncated_normal([784,enum1],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([enum1]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([enum1,enum2],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([enum2]) + 0.1)\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1,w2)+b2)\n",
    "w3 = tf.Variable(tf.truncated_normal([enum2,10 ],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer2, w3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过计算可知，隐层神经元数量取（300,400,500,600,700,800）其结果相差无几，此处设为500\n",
    "接下来添加正则项进行测试，使用l2正则,正则化惩罚因子分别取（1,0.1,0.01,0.001,0.0001,0.00001）进行计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "enum = 500\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "w1 = tf.Variable(tf.truncated_normal([784,enum],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([enum]) + 0.1)\n",
    "layer1 = tf.sigmoid(tf.matmul(x,w1)+b1)\n",
    "w2 = tf.Variable(tf.truncated_normal([enum,300],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "layer2 = tf.sigmoid(tf.matmul(layer1,w2)+b2)\n",
    "w3 = tf.Variable(tf.truncated_normal([300,10 ],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(layer2, w3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1135\n",
      "0.1009\n",
      "0.1028\n",
      "0.1135\n",
      "0.1028\n",
      "0.1135\n",
      "0.1028\n",
      "0.0982\n",
      "0.1009\n",
      "0.1028\n",
      "0.1028\n"
     ]
    }
   ],
   "source": [
    "r = 0.1\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) + tf.contrib.layers.l2_regularizer(r)(w1) + tf.contrib.layers.l2_regularizer(r)(w2)\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1028\n",
      "0.7101\n",
      "0.8412\n",
      "0.7485\n",
      "0.7411\n",
      "0.851\n",
      "0.7954\n",
      "0.8407\n",
      "0.803\n",
      "0.6023\n",
      "0.8339\n"
     ]
    }
   ],
   "source": [
    "r = 0.01\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) + tf.contrib.layers.l2_regularizer(r)(w1) + tf.contrib.layers.l2_regularizer(r)(w2)\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1135\n",
      "0.8784\n",
      "0.888\n",
      "0.9037\n",
      "0.9029\n",
      "0.9147\n",
      "0.922\n",
      "0.9189\n",
      "0.9165\n",
      "0.8734\n",
      "0.8494\n"
     ]
    }
   ],
   "source": [
    "r = 0.001\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) + tf.contrib.layers.l2_regularizer(r)(w1) + tf.contrib.layers.l2_regularizer(r)(w2)\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0957\n",
      "0.8954\n",
      "0.8963\n",
      "0.9077\n",
      "0.927\n",
      "0.923\n",
      "0.939\n",
      "0.9409\n",
      "0.9438\n",
      "0.9434\n",
      "0.9481\n"
     ]
    }
   ],
   "source": [
    "r = 0.0001\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) + tf.contrib.layers.l2_regularizer(r)(w1) + tf.contrib.layers.l2_regularizer(r)(w2)\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0974\n",
      "0.8752\n",
      "0.9062\n",
      "0.9108\n",
      "0.9165\n",
      "0.927\n",
      "0.9378\n",
      "0.9425\n",
      "0.9462\n",
      "0.9455\n",
      "0.9527\n"
     ]
    }
   ],
   "source": [
    "r = 0.00001\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) + tf.contrib.layers.l2_regularizer(r)(w1) + tf.contrib.layers.l2_regularizer(r)(w2)\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "for i in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    #每隔100个循环打印一次结果\n",
    "    if not i%300:\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上面的结果看，正则化的添加对结果并没有太多影响，将惩罚因子设为最优的0.00001,接下来调整学习率试试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========learning_rate=0.100000========\n",
      "0.1026\n",
      "0.8394\n",
      "0.8774\n",
      "0.8899\n",
      "0.8959\n",
      "0.9051\n",
      "0.9131\n",
      "0.9144\n",
      "0.9139\n",
      "0.912\n",
      "0.9204\n",
      "========learning_rate=0.300000========\n",
      "0.1135\n",
      "0.88\n",
      "0.9073\n",
      "0.915\n",
      "0.9167\n",
      "0.9221\n",
      "0.925\n",
      "0.9311\n",
      "0.9335\n",
      "0.9327\n",
      "0.9388\n",
      "========learning_rate=0.500000========\n",
      "0.101\n",
      "0.8893\n",
      "0.9139\n",
      "0.8946\n",
      "0.9203\n",
      "0.9333\n",
      "0.9315\n",
      "0.9439\n",
      "0.9488\n",
      "0.9478\n",
      "0.9506\n",
      "========learning_rate=0.700000========\n",
      "0.0982\n",
      "0.8956\n",
      "0.9022\n",
      "0.9034\n",
      "0.9349\n",
      "0.9366\n",
      "0.9433\n",
      "0.9513\n",
      "0.9543\n",
      "0.9579\n",
      "0.9571\n",
      "========learning_rate=0.900000========\n",
      "0.1051\n",
      "0.9002\n",
      "0.9211\n",
      "0.9295\n",
      "0.9447\n",
      "0.9492\n",
      "0.9535\n",
      "0.9553\n",
      "0.9585\n",
      "0.9627\n",
      "0.967\n"
     ]
    }
   ],
   "source": [
    "r = 0.00001\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) + tf.contrib.layers.l2_regularizer(r)(w1) + tf.contrib.layers.l2_regularizer(r)(w2)\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "rate = [0.1,0.3,0.5,0.7,0.9]\n",
    "for i,item in enumerate(rate):\n",
    "    sess = tf.Session()\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    print(\"========learning_rate=%f========\" %(item))\n",
    "    train_step = tf.train.GradientDescentOptimizer(item).minimize(cross_entropy)\n",
    "    for i in range(3000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "        if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "发现学习率越高得到的效果越好，将学习率继续调高试试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========learning_rate=1.200000========\n",
      "0.098\n",
      "0.9031\n",
      "0.9221\n",
      "0.9373\n",
      "0.9285\n",
      "0.9364\n",
      "0.9543\n",
      "0.9607\n",
      "0.965\n",
      "0.9637\n",
      "0.9664\n",
      "========learning_rate=1.500000========\n",
      "0.1135\n",
      "0.8567\n",
      "0.9191\n",
      "0.9427\n",
      "0.9535\n",
      "0.9534\n",
      "0.9627\n",
      "0.9665\n",
      "0.9656\n",
      "0.9671\n",
      "0.9686\n",
      "========learning_rate=1.800000========\n",
      "0.0892\n",
      "0.9131\n",
      "0.9183\n",
      "0.9411\n",
      "0.9571\n",
      "0.9586\n",
      "0.9622\n",
      "0.9633\n",
      "0.9676\n",
      "0.9699\n",
      "0.9705\n",
      "========learning_rate=2.100000========\n",
      "0.0982\n",
      "0.8912\n",
      "0.9356\n",
      "0.9481\n",
      "0.954\n",
      "0.9626\n",
      "0.9643\n",
      "0.9636\n",
      "0.9678\n",
      "0.9714\n",
      "0.9726\n",
      "========learning_rate=2.400000========\n",
      "0.1028\n",
      "0.8972\n",
      "0.9336\n",
      "0.9459\n",
      "0.9604\n",
      "0.958\n",
      "0.9661\n",
      "0.967\n",
      "0.9675\n",
      "0.9723\n",
      "0.9699\n"
     ]
    }
   ],
   "source": [
    "rate = [1.2,1.5,1.8,2.1,2.4]\n",
    "for i,item in enumerate(rate):\n",
    "    sess = tf.Session()\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    print(\"========learning_rate=%f========\" %(item))\n",
    "    train_step = tf.train.GradientDescentOptimizer(item).minimize(cross_entropy)\n",
    "    for i in range(3000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "        if not i%300:\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sigmoid激活函数的效果最高调到0.9726左右，接下来试试其它激活函数"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
